"""
Download the (metadata of) all the datasets of a CKAN repository.
The datasets are stored as JSON files.
The script waits 4 seconds between HTTP requests.
"""

__author__ = "Bart Hanssens <bart.hanssens@fedict.be>"

import os, errno, sys, time, argparse, logging
import urllib2, json

logging.basicConfig(filename='ckan-harvest.log', level=logging.DEBUG)


def getdata(url):
	"""Make a HTTP GET request."""
	logging.info('Getting ' + url)
	
	try:
		req = urllib2.Request(url)
		resp = urllib2.urlopen(req)
		return resp.read()
	except urllib2.HTTPError as e:
		logging.error('HTTP Error : ' + str(e.code))
		return ''

		
def ckanset(url, dataset, path):
	"""Get the datasets from CKAN repo and store them as separate JSON files."""
	file = os.path.join(path, dataset)
	
	if not os.path.exists(file):
		url = url + '/api/3/action/package_show?id=' + dataset
		data = getdata(url)

		if data:
			result = json.loads(data)
				
			with open(file, "w") as outfile:
				json.dump(result, outfile, indent=4)
		
			time.sleep(4)
	else:
		logging.info('Skipping ' + file)

		
def ckanlist(url):
	"""Get the list of all datasets available from the CKAN repo."""
	url = url + '/api/3/action/package_list'
	data = getdata(url)
	result = json.loads(data)
	return result['result']


def main():	
	parser = argparse.ArgumentParser(description='Download CKAN repo to JSON files')
	parser.add_argument('--url', help='CKAN url', required=True)
	parser.add_argument('--outdir', help='Output directory', required=True)
	args = parser.parse_args()

	for dataset in ckanlist(args.url):
		ckanset(args.url, dataset, args.outdir)

if __name__ == "__main__":
	main()
